package lesson;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.io.IOException;
import java.util.logging.Level;

public class TestJsoup {
    @Test
    public void test1() throws IOException {
        //1、根据URL加载document对象(网页)
        //2、根据document查找页面中的元素，并获取数据
        //3、通过id或者标签名称查找
        /**
         * Element content = doc.getElementById("content");
         * Elements links = content.getElementsByTag("a");
         */
        //4、还可以通过css选择器查找

       /* Document document = Jsoup.connect(
                "https://www.zhipin.com/c101010100/?query=Java&page=2&ka=page-2").get();*/
        //问题在与boss中有部分数据是异步加载
        //需要做特殊处理

        //System.out.println(ulElement.html());
        readData("c101010100","C",1);
        //https://www.zhipin.com/c100010000/?query=%E5%B5%8C%E5%85%A5%E5%BC%8F&page=2&ka=page-2

    }

    public void readData(String cityNum,String kw,Integer pageNum) throws IOException {
        String url = "https://www.zhipin.com/" +
                cityNum +
                "/?query=" +
                kw +
                "&page=" +
                pageNum +
                "&ka=page-" +
                pageNum;
        System.out.println(pageNum);
        //此处通过url加载document对象
        Document document = loadElement(url);
        Element ulElement = document.selectFirst("#main>.job-box>.job-list>ul");
        Elements allLiElement = ulElement.getElementsByTag("li");
        for(Element element:allLiElement){
            Element jobInfoElement = element.selectFirst(".info-primary");
            Element primaryBox = jobInfoElement.selectFirst(".primary-box");
            String dataJobid = primaryBox.attr("data-jobid");
            System.out.println(dataJobid);
            System.out.println(jobInfoElement.selectFirst(".job-name").text());
            System.out.println(jobInfoElement.selectFirst(".job-area-wrapper").text());
            System.out.println(jobInfoElement.selectFirst(".job-limit>span").text());
            System.out.println(jobInfoElement.selectFirst(".company-text>.name").text());
            //System.out.println(jobInfoElement.selectFirst(".job-limit>p").text());
            String[] strs = jobInfoElement.selectFirst(".job-limit>p").text().trim().split(" ");
            System.out.println(strs[0]);
            System.out.println(strs[1]);
            System.out.println("----------------");
        }

        if (document.getElementsByClass("page").size() > 0) {
            Element pagediv = document.getElementsByClass("page").get(0);
            Element nexta = pagediv.getElementsByClass("next").get(0);
            if (nexta.attr("href").contains("javascript")) {
                System.out.println("数据爬取完毕");
            } else {
                // 还可以继续爬取数据
                readData(cityNum, kw, pageNum + 1);
            }
        }

    }

    public Document loadElement(String url) throws IOException {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //屏蔽日志信息
        LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
                "org.apache.commons.logging.impl.NoOpLog");
        java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
        //支持JavaScript
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setActiveXNative(false);
        // webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setTimeout(5000);

        System.out.println(url);
        HtmlPage rootPage = webClient.getPage(url);
        //设置一个运行JavaScript的时间
        webClient.waitForBackgroundJavaScript(5000);
        String html = rootPage.asXml();

        Document document = Jsoup.parse(html);
        return document;
    }
}
